{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COMP551 Mini Project 2 - IMDB Sentiment Analysis  \n",
    "This is the codes of mini project2 - IMDB Sentiment Analysis.  \n",
    "\n",
    "## AUTHORS\n",
    "Pengnan Fan, ID#260768510  \n",
    "Qifei Zhao, ID#260719382  \n",
    "\n",
    "## TASKS\n",
    "1. **Bernoulli Naive Bayes** (w/o any external library).  \n",
    "2. **At least 2** out of 3 classifiers from the SciKit. i.e. suggestions: logistic regression, decision tree, or support vector machines  \n",
    "3. **At least 2** different features extraction pipelines for processing the data. \n",
    "4. A model validation. i.e. **K-fold cross validation**  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 0 - Preparation  \n",
    "by Pengnan, Sherry, and Kaylee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 0.1 - List of Packages Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import sys\n",
    "import time\n",
    "import math\n",
    "from time import sleep\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import nltk\n",
    "import numpy\n",
    "import scipy\n",
    "import sklearn.datasets\n",
    "import contractions\n",
    "from itertools import groupby\n",
    "import string\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import csv\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn import tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 0.2 - Data Loading Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# Please add your address here as string\n",
    "ADDRESS_TRAIN_PENGNAN = \"D:\\\\McGill\\\\19Fall\\\\COMP 551\\\\Projects\\\\Project2\\\\comp-551-imbd-sentiment-classification\\\\train\"\n",
    "ADDRESS_TEST_PENGNAN = \"D:\\\\McGill\\\\19Fall\\\\COMP 551\\\\Projects\\\\Project2\\\\comp-551-imbd-sentiment-classification\\\\test\"\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# @author Pengnan Fan\n",
    "# @param address of train set\n",
    "# @return a dict of list of dict\n",
    "def loadData(address):\n",
    "    print(\"Start loading negative set\")\n",
    "    neg = sklearn.datasets.load_files(address, categories={\"neg\"})\n",
    "    print(\"Complete loading negative set\")\n",
    "    print(\"Start loading positive set\")\n",
    "    pos = sklearn.datasets.load_files(address, categories={\"pos\"})\n",
    "    print(\"Complete loading postive set\")\n",
    "    \n",
    "    negSet = list()\n",
    "    count = 0\n",
    "    size = len(neg.data)\n",
    "    for x in neg.data:\n",
    "        negSet.append({\"comment\":x.decode('utf-8'), \"isPos\":0})\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(size)) * 100)\n",
    "        sys.stdout.write(\"Preparing negative set: [%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    print()\n",
    "    \n",
    "    posSet = list()\n",
    "    count = 0\n",
    "    size = len(pos.data)\n",
    "    for x in pos.data:\n",
    "        posSet.append({\"comment\":x.decode('utf-8'), \"isPos\":1})\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(size)) * 100)\n",
    "        sys.stdout.write(\"Prepare positive set: [%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    output = {'pos':posSet, 'neg':negSet, 'all':posSet+negSet}\n",
    "    print(\"\\nFinish preparing\")\n",
    "    return output\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# Path\n",
    "train_path=\"D:\\\\McGill\\\\19Fall\\\\COMP 551\\\\Projects\\\\Project2\\\\comp-551-imbd-sentiment-classification\\\\train\"\n",
    "test_path=\"D:\\\\McGill\\\\19Fall\\\\COMP 551\\\\Projects\\\\Project2\\\\comp-551-imbd-sentiment-classification\\\\test\"\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# This function loads train data to a list\n",
    "# @author\n",
    "# @param class_name: either \"pos\" for loading positive data and otherwise loading negative data\n",
    "# @return data: list of train data\n",
    "def load_train(class_name):\n",
    "    label=0\n",
    "    if class_name is \"pos\":\n",
    "        print(\"Start loading positive data\")\n",
    "        label=1\n",
    "    else:\n",
    "        print(\"Start loading negative data\")\n",
    "    data=[]\n",
    "    count = 0\n",
    "    lenW = 12500\n",
    "    for file in glob.glob(train_path+\"/\"+class_name+\"/*.txt\"):\n",
    "        f = open(file, \"r\")\n",
    "        data.append([f.read(),label])\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    if class_name is \"pos\":\n",
    "        print(\"\\nComplete loading positive data\")\n",
    "    else:\n",
    "        print(\"\\nComplete loading negative data\")\n",
    "    return data\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# This function loads test data to a list\n",
    "# @author\n",
    "# @return data: list of test data\n",
    "def load_test():\n",
    "    data=[]\n",
    "    print(\"Start loading test data\")\n",
    "    count = 0\n",
    "    lenW = 25000\n",
    "    for i in range(0,25000):\n",
    "        file=test_path+\"/\"+str(i)+\".txt\"\n",
    "        f = open(file, \"r\")\n",
    "        data.append([f.read(),i])\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete loading test data\")\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 1 - Bernoulli Naive Bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 1.1 - Bernoulli Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set for prediction\n",
    "# @param wordExistance: dict {'pos' -> pos word existance, 'neg' -> neg word existance, 'all' -> all word existance}\n",
    "# @param wordSet: list of unique words\n",
    "# @param size: dict {'pos' -> pos size, 'neg' -> neg size, 'all' -> all size}\n",
    "# @return prediction\n",
    "def bernoulliNaiveBayes(dataSet, wordExistance, wordSet, size):\n",
    "    pPos = size['pos']/size['all']\n",
    "    pNeg = size['neg']/size['all']\n",
    "    prediction = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    print(\"Start Bernoulli naive Bayes classifying\")\n",
    "    for exp in dataSet:\n",
    "        #pPos_x = numpy.log([pPos])\n",
    "        #pNeg_x = numpy.log([pNeg])\n",
    "        \n",
    "        pred = numpy.log([pPos/nNeg])\n",
    "        \n",
    "        # Calculating P(Y|X)\n",
    "        for word in wordSet:\n",
    "            pos = 0\n",
    "            neg = 0\n",
    "            \n",
    "            if word in exp:\n",
    "                pos = (wordExistance['pos'][word]+1)/(size['pos']+2)\n",
    "                neg = (wordExistance['neg'][word]+1)/(size['neg']+2)\n",
    "            else:\n",
    "                pos = 1 - ((wordExistance['pos'][word]+1)/(size['pos']+2))\n",
    "                neg = 1 - ((wordExistance['neg'][word]+1)/(size['neg']+2))\n",
    "            \n",
    "            #pPos_x += numpy.log([pos])\n",
    "            #pNeg_x += numpy.log([neg])\n",
    "            pred+=numpy.log([pos])\n",
    "        \n",
    "        # Logistic decision boundary\n",
    "        #log_ratio = pPos_x - pNeg_x\n",
    "        \n",
    "        if pred > 0:\n",
    "            prediction.append(1)\n",
    "        else:\n",
    "            prediction.append(0)\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    print(\"\\nComplete Bernoulli naive Bayes classifying\")\n",
    "    \n",
    "    return prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 1.2 - Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: train set with label\n",
    "# @param prediction: prediction of each example in the trainSet\n",
    "# @return result: list of results: {TP: true pos, TN: true neg, FP: false pos, FN: false neg}\n",
    "def evaluation(dataSet, prediction):\n",
    "    size = len(prediction)\n",
    "    count = 0\n",
    "    result = {'TP':0,'TN':0,'FP':0,'FN':0}\n",
    "    \n",
    "    print(\"Start evaluating classification\")\n",
    "    for i in range(size):\n",
    "        if prediction[i]==1:\n",
    "            if dataSet[i]['isPos']==1:\n",
    "                result['TP']+=1\n",
    "            else:\n",
    "                result['FP']+=1\n",
    "        else:\n",
    "            if dataSet[i]['isPos']==1:\n",
    "                result['FN']+=1\n",
    "            else:\n",
    "                result['TN']+=1\n",
    "                \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(size)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete evaluating classification\")\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 2 - Logistic Regression, Support Vector Machine, and Decision Tree  \n",
    "by Sherry and Kaylee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2.1 - Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (train_data, 0.8, 0.2, True, False, 1, 3, 100)\n",
    "def LogReg(train_data, train_ratio, test_ratio, random, tfidf, minN, maxN, c):\n",
    "    train_set,valid_set=sklearn.model_selection.train_test_split(train_data,train_size=train_ratio,test_size=test_ratio,shuffle=random)\n",
    "    train_x=[comment[0] for comment in train_set]\n",
    "    train_y=[comment[1] for comment in train_set]\n",
    "    valid_x=[comment[0] for comment in valid_set]\n",
    "    valid_y=[comment[1] for comment in valid_set]\n",
    "    \n",
    "    vectorizer = CountVectorizer(ngram_range=(minN, maxN))\n",
    "    X = vectorizer.fit_transform(train_x)\n",
    "    \n",
    "    tfidf_transformer = TfidfTransformer(use_idf=tfidf)\n",
    "    X=tfidf_transformer.fit_transform(X)\n",
    "    \n",
    "    lg=LogisticRegression(C=c)\n",
    "    start_learn = time.time()\n",
    "    lg.fit(X,train_y)\n",
    "    end_learn = time.time()\n",
    "    #result = sklearn.model_selection.cross_validate(lg, X, y_train, cv=5, return_train_score=True)\n",
    "    x_v=vectorizer.transform(valid_x)\n",
    "    start_pred = time.time()\n",
    "    y_pred = lg.predict(x_v)\n",
    "    end_pred = time.time()\n",
    "    \n",
    "    score = sklearn.metrics.accuracy_score(valid_y, y_pred)\n",
    "    print (\"LR Accuracy: \", score)\n",
    "    print (\"Learning Running Time: \", end_learn - start_learn, \"seconds\")\n",
    "    print (\"Predicting Running Time: \", end_pred - start_pred, \"seconds\")\n",
    "    return score\n",
    "    \n",
    "    '''\n",
    "    print (\"LR Train Accuracy = \", result['train_score'])\n",
    "    print(\"Avg Train Accuracy = \", numpy.mean(result['train_score']))\n",
    "    print (\"LR Cross Validation Accuracy = \", result['test_score'])\n",
    "    print(\"Avg Validation Accuracy = \", numpy.mean(result['test_score']))\n",
    "    '''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2.2 Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SVM(train_data, train_ratio, test_ratio, random, tfidf, minN, maxN, c):\n",
    "    train_set,valid_set=sklearn.model_selection.train_test_split(train_data,train_size=train_ratio,test_size=test_ratio,shuffle=random)\n",
    "    train_x=[comment[0] for comment in train_set]\n",
    "    train_y=[comment[1] for comment in train_set]\n",
    "    valid_x=[comment[0] for comment in valid_set]\n",
    "    valid_y=[comment[1] for comment in valid_set]\n",
    "    \n",
    "    vectorizer = CountVectorizer(ngram_range=(minN, maxN))\n",
    "    X = vectorizer.fit_transform(train_x)\n",
    "    \n",
    "    tfidf_transformer = TfidfTransformer(use_idf=tfidf)\n",
    "    X=tfidf_transformer.fit_transform(X)\n",
    "    \n",
    "    \n",
    "    clf=SVC(kernel='linear', C=c)\n",
    "    start_learn = time.time()\n",
    "    clf.fit(X,train_y)\n",
    "    end_learn = time.time()\n",
    "    x_v=vectorizer.transform(valid_x)\n",
    "    start_pred = time.time()\n",
    "    y_pred=clf.predict(x_v)\n",
    "    end_pred = time.time()\n",
    "    score = sklearn.metrics.accuracy_score(valid_y, y_pred)\n",
    "    print (\"SVM Accuracy: \", score)\n",
    "    print (\"Learning Running Time: \", end_learn - start_learn, \"seconds\")\n",
    "    print (\"Predicting Running Time: \", end_pred - start_pred, \"seconds\")\n",
    "    return score\n",
    "    #result = sklearn.model_selection.cross_validate(clf, X, y_train, cv=5, return_train_score=True)\n",
    "    '''\n",
    "    print (\"SVM Train Accuracy = \", result['train_score'])\n",
    "    print(\"Avg Train Accuracy = \", numpy.mean(result['train_score']))\n",
    "    print (\"SVM Cross Validation Accuracy = \", result['test_score'])\n",
    "    print(\"Avg Validation Accuracy = \", numpy.mean(result['test_score']))\n",
    "    '''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2.3 Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DecTree(train_data, train_ratio, test_ratio, random, tfidf, minN, maxN):\n",
    "    train_set,valid_set=sklearn.model_selection.train_test_split(train_data,train_size=train_ratio,test_size=test_ratio,shuffle=random)\n",
    "    train_x=[comment[0] for comment in train_set]\n",
    "    train_y=[comment[1] for comment in train_set]\n",
    "    valid_x=[comment[0] for comment in valid_set]\n",
    "    valid_y=[comment[1] for comment in valid_set]\n",
    "    \n",
    "    \n",
    "    vectorizer = CountVectorizer(ngram_range=(minN, maxN))\n",
    "    X = vectorizer.fit_transform(train_x)\n",
    "    \n",
    "    tfidf_transformer = TfidfTransformer(use_idf=tfidf)\n",
    "    X=tfidf_transformer.fit_transform(X)\n",
    "     \n",
    "\n",
    "    dt = tree.DecisionTreeClassifier(max_depth=10,min_samples_leaf=2,max_leaf_nodes=300,min_samples_split=2)\n",
    "    start_learn = time.time()\n",
    "    dt.fit(X, train_y)\n",
    "    end_learn = time.time()\n",
    "    x_v=vectorizer.transform(valid_x)\n",
    "    start_pred = time.time()\n",
    "    y_pred = dt.predict(x_v)\n",
    "    end_pred = time.time()\n",
    "    score = sklearn.metrics.accuracy_score(valid_y, y_pred)\n",
    "    print (\"Decision Tree Accuracy: \", score)\n",
    "    print (\"Learning Running Time: \", end_learn - start_learn, \"seconds\")\n",
    "    print (\"Predicting Running Time: \", end_pred - start_pred, \"seconds\")\n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 3 - Feature Extraction Pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.1 - Word Frequency without Stopword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @acknowledgement Yuxiang Ma, for this function is edited based on his in miniproject1\n",
    "# @param dataSet: set of comments\n",
    "# @return naiveCount: word frequency without stopwords\n",
    "def wordsFrequencyStopword(dataSet):\n",
    "    stopwordCount = dict()\n",
    "    totalString = str()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['pos'])\n",
    "    \n",
    "    print(\"Start counting naive word frequency of positive set\")\n",
    "    \n",
    "    for comment in dataSet['pos']:\n",
    "        totalString = totalString + ' ' + comment['comment'].lower()\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    withoutPunc = tokenizer.tokenize(totalString)\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    posCountDict = Counter(s.lower() for s in withoutPunc if s.lower() not in stopwordsSet)\n",
    "    print(\"\\nComplete counting naive word frequency of positive set\")\n",
    "    \n",
    "    totalString = str()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['neg'])\n",
    "    \n",
    "    print(\"Start counting naive word frequency of negative set\")\n",
    "    \n",
    "    for comment in dataSet['neg']:\n",
    "        totalString = totalString + ' ' + comment['comment'].lower()\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    withoutPunc = tokenizer.tokenize(totalString)\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    negCountDict = Counter(s.lower() for s in withoutPunc if s.lower() not in stopwordsSet)\n",
    "    print(\"\\nComplete counting naive word frequency of negative set\")\n",
    "    \n",
    "    totalString = str()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['all'])\n",
    "    \n",
    "    print(\"Start counting naive word frequency of all set\")\n",
    "    \n",
    "    for comment in dataSet['all']:\n",
    "        totalString = totalString + ' ' + comment['comment'].lower()\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    withoutPunc = tokenizer.tokenize(totalString)\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    allCountDict = Counter(s.lower() for s in withoutPunc if s.lower() not in stopwordsSet)\n",
    "    print(\"\\nComplete counting naive word frequency of all set\")\n",
    "    \n",
    "    stopwordCount = {'pos':posCountDict, 'neg':negCountDict, 'all':allCountDict}\n",
    "    \n",
    "    return stopwordCount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.2 - Number of Naive Existance of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set of comments\n",
    "# @return naiveCount: list of words of num of naive existances\n",
    "def numOfExistanceNaive(dataSet):\n",
    "    naiveCount = dict()\n",
    "    totalComments = []\n",
    "    count = 0\n",
    "    lenW = len(dataSet['pos'])\n",
    "    \n",
    "    print(\"Start counting number of naive word existance of positive set\")\n",
    "    for comment in dataSet['pos']:\n",
    "        commentSplit = comment['comment'].lower().split(\" \")\n",
    "        wordsToAdd = []\n",
    "        for word in commentSplit:\n",
    "            if word not in wordsToAdd:\n",
    "                wordsToAdd.append(word)\n",
    "        totalComments+=wordsToAdd\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    posCount = Counter(x for x in totalComments)\n",
    "    print(\"\\nComplete counting number of naive word existance of positive set\")\n",
    "    \n",
    "    totalComments = []\n",
    "    count = 0\n",
    "    lenW = len(dataSet['neg'])\n",
    "    \n",
    "    print(\"Start counting number of naive word existance of negative set\")\n",
    "    for comment in dataSet['neg']:\n",
    "        commentSplit = comment['comment'].lower().split(\" \")\n",
    "        wordsToAdd = []\n",
    "        for word in commentSplit:\n",
    "            if word not in wordsToAdd:\n",
    "                wordsToAdd.append(word)\n",
    "        totalComments+=wordsToAdd\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    negCount = Counter(x for x in totalComments)\n",
    "    print(\"\\nComplete counting number of naive word existance of negative set\")\n",
    "    \n",
    "    totalComments = []\n",
    "    count = 0\n",
    "    lenW = len(dataSet['all'])\n",
    "    \n",
    "    print(\"Start counting number of naive word existance of all set\")\n",
    "    for comment in dataSet['all']:\n",
    "        commentSplit = comment['comment'].lower().split(\" \")\n",
    "        wordsToAdd = []\n",
    "        for word in commentSplit:\n",
    "            if word not in wordsToAdd:\n",
    "                wordsToAdd.append(word)\n",
    "        totalComments+=wordsToAdd\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    allCount = Counter(x for x in totalComments)\n",
    "    print(\"\\nComplete counting number of naive word existance of all set\")\n",
    "    \n",
    "    naiveCount = {'pos':posCount, 'neg':negCount, 'all':allCount}\n",
    "    \n",
    "    return naiveCount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.3.1 - Number of Existance of Words without Stopwords, Duplicates, and with Stemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set of comments\n",
    "# @return naiveCount: list of num of existances of words without stopwords, duplicates, and with stemmer\n",
    "def advancedNumOfExistance(dataSet):\n",
    "    ps = PorterStemmer()\n",
    "    countDict = dict()\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    stopwordsSet.add('br')\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['pos'])\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of positive set\")\n",
    "    for exp in dataSet['pos']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in sent_clean.split():\n",
    "            word = ps.stem(word)\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    posDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of positive set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['neg'])\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of negative set\")\n",
    "    for exp in dataSet['neg']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in sent_clean.split():\n",
    "            word = ps.stem(word)\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    negDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of negative set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['all'])\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of all set\")\n",
    "    for exp in dataSet['all']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in sent_clean.split():\n",
    "            word = ps.stem(word)\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1     \n",
    "    \n",
    "    allDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of all set\")\n",
    "    \n",
    "    countDict = {'neg':negDict, 'pos':posDict, 'all':allDict}\n",
    "    \n",
    "    return countDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.3.2 - With Duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set of comments\n",
    "# @return naiveCount: list of num of existances of words without stopwords and with stemmer\n",
    "def advancedNumOfExistanceWithDuplicates(dataSet):\n",
    "    output = dict()\n",
    "    countDict = dict()\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    stopwordsSet.add('br')\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['pos'])\n",
    "    eachFreqPos = list()\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of positive set\")\n",
    "    for exp in dataSet['pos']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        # processedComment = ([k for k, v in groupby(sent_clean.split())])\n",
    "        \n",
    "        wordSet+=sent_clean.split()\n",
    "        \n",
    "        temp_pos = Counter(s for s in sent_clean.split() if s not in stopwordsSet)\n",
    "        \n",
    "        eachFreqPos.append(temp_pos)\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    posDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of positive set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['neg'])\n",
    "    eachFreqNeg = list()\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of negative set\")\n",
    "    for exp in dataSet['neg']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        # processedComment = ([k for k, v in groupby(sent_clean.split())])\n",
    "                \n",
    "        wordSet+=sent_clean.split()\n",
    "        \n",
    "        temp_neg = Counter(s for s in sent_clean.split() if s not in stopwordsSet)\n",
    "        \n",
    "        eachFreqNeg.append(temp_neg)\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    negDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of negative set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['all'])\n",
    "    \n",
    "    print(\"Start counting number of words without stopwords of existance of all set\")\n",
    "    for exp in dataSet['all']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        # processedComment = ([k for k, v in groupby(sent_clean.split())])\n",
    "        \n",
    "        wordSet+=sent_clean.split()\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1     \n",
    "    \n",
    "    allDict = Counter(s for s in wordSet if s not in stopwordsSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of words without stopwords of existance of all set\")\n",
    "    \n",
    "    countDict = {'neg':negDict, 'pos':posDict, 'all':allDict}\n",
    "    \n",
    "    eachFreq = eachFreqPos+eachFreqNeg\n",
    "    \n",
    "    output = {'allFreq':countDict, 'individual':eachFreq}\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.3.3 - Number of Existance of N-gram Words without Stopwords, Duplicates and with Stemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set of comments\n",
    "# @param n: used for n-gram\n",
    "# @return countDict: list of num of existances of words without stopwords, duplicates and with stemmer\n",
    "def nGram(dataSet, n):\n",
    "    ps = PorterStemmer()\n",
    "    countDict = dict()\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    stopwordsSet.add('br')\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['pos'])\n",
    "    \n",
    "    print(\"Start counting number of \", n, \"-grams without duplicates of positive set\")\n",
    "    for exp in dataSet['pos']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        withoutStopwords = list()\n",
    "        for x in sent_clean.split():\n",
    "            if x not in stopwordsSet:\n",
    "                withoutStopwords.append(ps.stem(x))\n",
    "        bi = list(nltk.ngrams(withoutStopwords, n))\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in bi:\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    posDict = Counter(s for s in wordSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of \", n, \"-grams without duplicates of positive set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['neg'])\n",
    "    \n",
    "    print(\"Start counting number of \", n, \"-grams without duplicates of negative set\")\n",
    "    for exp in dataSet['neg']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        withoutStopwords = list()\n",
    "        for x in sent_clean.split():\n",
    "            if x not in stopwordsSet:\n",
    "                withoutStopwords.append(ps.stem(x))\n",
    "        bi = list(nltk.ngrams(withoutStopwords, n))\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in bi:\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    \n",
    "    negDict = Counter(s for s in wordSet)\n",
    "    \n",
    "    print(\"\\nComplete counting number of \", n, \"-grams without duplicates of negative set\")\n",
    "    \n",
    "    wordSet = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet['all'])\n",
    "    \n",
    "    print(\"Start counting number of \", n, \"-grams without duplicates of all set\")\n",
    "    for exp in dataSet['all']: \n",
    "        comment = contractions.fix(exp['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        withoutStopwords = list()\n",
    "        for x in sent_clean.split():\n",
    "            if x not in stopwordsSet:\n",
    "                withoutStopwords.append(ps.stem(x))\n",
    "        bi = list(nltk.ngrams(withoutStopwords, n))\n",
    "        processedComment = []\n",
    "        \n",
    "        for word in bi:\n",
    "            if word not in processedComment:\n",
    "                processedComment.append(word)\n",
    "                \n",
    "        wordSet+=processedComment\n",
    "            \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1     \n",
    "    \n",
    "    allDict = Counter(s for s in wordSet)\n",
    "    \n",
    "    print(\"\\nCounting counting number of \", n, \"-grams without duplicates of all set\")\n",
    "    \n",
    "    countDict = {'neg':negDict, 'pos':posDict, 'all':allDict}\n",
    "    \n",
    "    return countDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.3.4 - CountAllWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet: set used to learn\n",
    "# @return output: list of unique words shown in dataSet\n",
    "def allWords(dataSet):\n",
    "    output = list()\n",
    "    lenW = len(dataSet)\n",
    "    count = 0\n",
    "    print(\"Start counting all words\")\n",
    "    for comments in dataSet:\n",
    "        for word in comments:\n",
    "            if word not in output:\n",
    "                output.append(word)\n",
    "                \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    print(\"\\nComplete counting all words\")\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3.4 - Data Standardization and Clean-up Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @author Pengnan Fan\n",
    "# @param dataSet:comment data need to be proceed\n",
    "# @return proceedData\n",
    "def dataStandardization(dataSet):\n",
    "    proceedData = list()\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    stopwordsSet.add('br')\n",
    "    \n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    \n",
    "    print(\"Start proceeding data\")\n",
    "    for data in dataSet: \n",
    "        comment = contractions.fix(data['comment'].lower())\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        processedComment = sent_clean.split()\n",
    "        toAdd = list()\n",
    "        for x in processedComment:\n",
    "            if x not in stopwordsSet and x not in toAdd:\n",
    "                toAdd.append(x)\n",
    "                \n",
    "        proceedData.append(toAdd)\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "        \n",
    "    print(\"\\nComplete proceeding data\")\n",
    "    return proceedData\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# @author Pengnan\n",
    "# @param dataSet:set of data with comments\n",
    "# @return extendedSentence:set of comments without abbr. i.e. it's -> it is\n",
    "def extendSentences(dataSet):\n",
    "    extenedSentence = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    print(\"Start extending sentences\")\n",
    "    for data in dataSet:\n",
    "        extenedSentence.append([contractions.fix(data[0].lower()),data[1]])\n",
    "        \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete extending sentences\")\n",
    "    return extenedSentence\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# @author Pengnan\n",
    "def removePunctuationAndStopwords(dataSet, additionalStopwords=[]):\n",
    "    clean = list()\n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    stopwordsSet = set(stopwords.words())\n",
    "    print(\"Start removing punctuation and stopwords\")\n",
    "    for data in dataSet:\n",
    "        comment = data[0].lower()\n",
    "        sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "        sent_clean = comment.translate(sent_map)\n",
    "        processedComment = sent_clean.split()\n",
    "        fixed = str()\n",
    "        for word in processedComment:\n",
    "            if word not in stopwordsSet and word not in additionalStopwords:\n",
    "                fixed = fixed + \" \" + word\n",
    "        clean.append([fixed,data[1]])\n",
    "    \n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete removing punctuation and stopwords\")\n",
    "    return clean\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "def lemmatize(dataSet):\n",
    "    processed = list()\n",
    "    lemm = WordNetLemmatizer()\n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    print(\"Start lemmatizing\")\n",
    "    for data in dataSet:\n",
    "        words = word_tokenize(data[0]) # word_tokenize() takes care of stripping too.\n",
    "        clean_text = str()\n",
    "        for word in words:\n",
    "            w = lemm.lemmatize(word)\n",
    "            clean_text += w + \" \"\n",
    "        processed.append([clean_text, data[1]])\n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete lemmatizing\")\n",
    "    return processed\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "def removeDuplicates(dataSet):\n",
    "    processed = list()\n",
    "    lemm = WordNetLemmatizer()\n",
    "    count = 0\n",
    "    lenW = len(dataSet)\n",
    "    print(\"Start removing duplicates\")\n",
    "    for data in dataSet:\n",
    "        words = word_tokenize(data[0]) # word_tokenize() takes care of stripping too.\n",
    "        noDup = list()\n",
    "        for word in words:\n",
    "            if word not in noDup:\n",
    "                noDup.append(word)\n",
    "                \n",
    "        clean_text = str()\n",
    "        for w in noDup:\n",
    "            clean_text = clean_text + \" \" + w\n",
    "        processed.append([clean_text, data[1]])\n",
    "        # This is for loading bar\n",
    "        sys.stdout.write('\\r')\n",
    "        c = int((float(count) / float(lenW)) * 100)\n",
    "        sys.stdout.write(\"[%-20s] %d%%\" % ('='*int(c / 5), c))\n",
    "        sleep(0.001)\n",
    "        sys.stdout.flush()\n",
    "        count += 1\n",
    "    print(\"\\nComplete removing duplicates\")\n",
    "    return processed\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# @author Qifei, Pengnan\n",
    "# @param text: a string of comment\n",
    "# @param custom_stopwords:list of additional stopwords\n",
    "# @return clean_txt: a string of clean-up comment\n",
    "def cleanUp(text, custom_stopwords=[]):\n",
    "    # Initilaise Lemmatizer object:\n",
    "    lemm = WordNetLemmatizer()\n",
    "    \n",
    "    # Extend words\n",
    "    comment = contractions.fix(text.lower())\n",
    "    sent_map = comment.maketrans(dict.fromkeys(string.punctuation))\n",
    "    sent_clean = comment.translate(sent_map)\n",
    "    \n",
    "    # Load NLTK stopwords:\n",
    "    my_stopwords = stopwords.words('english') + custom_stopwords\n",
    "    \n",
    "    clean_text = ''\n",
    "    \n",
    "    words = word_tokenize(sent_clean) # word_tokenize() takes care of stripping too.\n",
    "    \n",
    "    for word in words:\n",
    "        w = lemm.lemmatize(w)\n",
    "        if w not in my_stopwords and len(w)>2:\n",
    "            clean_text += w + \" \"\n",
    "    \n",
    "    return clean_text\n",
    "\n",
    "#-------------------------------------------------------------------------------------------------------------------------------------------#\n",
    "# @author\n",
    "# @param dataset: list of comments\n",
    "def cleanUp_data(dataset):\n",
    "    for i in range(0,len(dataset)):\n",
    "        comment=cleanUp(dataset[i][0])\n",
    "        dataset[i][0]=comment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crossValidation(mode, X,y_train, fold, c, train_score):\n",
    "    if mode is \"LR\":\n",
    "        lg=LogisticRegression(C=c)\n",
    "        lg.fit(X,y_train)\n",
    "        result = sklearn.model_selection.cross_validate(lg, X,y_train, cv=fold, return_train_score=train_score)\n",
    "        print (\"Logistic Regression Train Accuracy = \", result['train_score'])\n",
    "        print(\"Avg Train Accuracy = \", numpy.mean(result['train_score']))\n",
    "        print (\"Logistic Regression Cross Validation Accuracy = \", result['test_score'])\n",
    "        print(\"Avg Validation Accuracy = \", numpy.mean(result['test_score']))\n",
    "        return result\n",
    "    \n",
    "    elif mode is \"SVM\":\n",
    "        clf=SVC(kernel='linear', C=c)\n",
    "        clf.fit(X,y_train)\n",
    "        result = sklearn.model_selection.cross_validate(clf, X, y_train, cv=fold, return_train_score=train_score)\n",
    "        print (\"SVM Train Accuracy = \", result['train_score'])\n",
    "        print(\"Avg Train Accuracy = \", numpy.mean(result['train_score']))\n",
    "        print (\"SVM Cross Validation Accuracy = \", result['test_score'])\n",
    "        print(\"Avg Validation Accuracy = \", numpy.mean(result['test_score']))\n",
    "        return result\n",
    "    elif mode is \"DT\":\n",
    "        dt = tree.DecisionTreeClassifier(max_depth=10,min_samples_leaf=2,max_leaf_nodes=300,min_samples_split=2)\n",
    "        dt.fit(X, y_train)\n",
    "        result = sklearn.model_selection.cross_validate(dt, X, y_train, cv=fold, return_train_score=train_score)\n",
    "    else:\n",
    "        print(\"INVALID INPUTS\")\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Preparing train set and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start loading positive data\n",
      "[=================== ] 99%\n",
      "Complete loading positive data\n",
      "Start loading negative data\n",
      "[=================== ] 99%\n",
      "Complete loading negative data\n",
      "Start loading test data\n",
      "[=================== ] 99%\n",
      "Complete loading test data\n"
     ]
    }
   ],
   "source": [
    "train_data=load_train(\"pos\")+load_train(\"neg\")\n",
    "test_data=load_test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Selecting Model Based on Accuracy and Runing Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Accuracy:  0.8502\n",
      "Learning Running Time:  1.4989209175109863 seconds\n",
      "Predicting Running Time:  0.004904508590698242 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8502"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LogReg(train_data, 0.8, 0.2, True, False, 1, 1, 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy:  0.8812\n",
      "Learning Running Time:  341.59919571876526 seconds\n",
      "Predicting Running Time:  84.45325541496277 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8812"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM(train_data, 0.8, 0.2, True, False, 1, 1, 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Accuracy:  0.6848\n",
      "Learning Running Time:  10.170814037322998 seconds\n",
      "Predicting Running Time:  0.006982088088989258 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6848"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DecTree(train_data, 0.8, 0.2, True, False, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selectRegularizationParameter(mode, dataSet, minN, maxN, cv, tfidf, cMin, cMax):\n",
    "    train_x=[comment[0] for comment in dataSet]\n",
    "    train_y=[comment[1] for comment in dataSet]\n",
    "    output = list()\n",
    "    bestScore = 0\n",
    "    bestParameter = 0\n",
    "    \n",
    "    vectorizer = CountVectorizer(ngram_range=(minN, maxN))\n",
    "    X = vectorizer.fit_transform(train_x)\n",
    "    \n",
    "    tfidf_transformer = TfidfTransformer(use_idf=tfidf)\n",
    "    X=tfidf_transformer.fit_transform(X)\n",
    "    for i in range(cMin,cMax):\n",
    "        print(\"< Cross Validation\", 10**i, \">\")\n",
    "        result = crossValidation(mode, X,train_y, cv, 10**i, True)\n",
    "        score = numpy.mean(result['test_score'])\n",
    "        output.append([score])\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "< Cross Validation 0.001 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.6759  0.67935 0.6752  0.67605 0.67585]\n",
      "Avg Train Accuracy =  0.67647\n",
      "Logistic Regression Cross Validation Accuracy =  [0.6838 0.6676 0.669  0.661  0.6746]\n",
      "Avg Validation Accuracy =  0.6712\n",
      "< Cross Validation 0.01 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.7073  0.70955 0.7077  0.7073  0.7081 ]\n",
      "Avg Train Accuracy =  0.70799\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7152 0.6986 0.702  0.6922 0.7016]\n",
      "Avg Validation Accuracy =  0.7019200000000001\n",
      "< Cross Validation 0.1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.7964  0.79405 0.7972  0.7959  0.7948 ]\n",
      "Avg Train Accuracy =  0.7956700000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7882 0.7854 0.7812 0.7804 0.7828]\n",
      "Avg Validation Accuracy =  0.7836000000000001\n",
      "< Cross Validation 1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.87515 0.87805 0.87735 0.87565 0.87645]\n",
      "Avg Train Accuracy =  0.87653\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8506 0.8454 0.8444 0.844  0.8518]\n",
      "Avg Validation Accuracy =  0.84724\n",
      "< Cross Validation 10 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.93995 0.941   0.9409  0.9396  0.9383 ]\n",
      "Avg Train Accuracy =  0.93995\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8714 0.8582 0.8566 0.8636 0.8744]\n",
      "Avg Validation Accuracy =  0.8648399999999998\n",
      "< Cross Validation 100 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.9906  0.9919  0.99185 0.99155 0.99035]\n",
      "Avg Train Accuracy =  0.99125\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8604 0.8326 0.8412 0.856  0.8626]\n",
      "Avg Validation Accuracy =  0.8505600000000001\n",
      "< Cross Validation 1000 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1.      1.      0.99995 0.99995 1.     ]\n",
      "Avg Train Accuracy =  0.9999800000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8468 0.816  0.8246 0.8384 0.8522]\n",
      "Avg Validation Accuracy =  0.8356\n"
     ]
    }
   ],
   "source": [
    "LR_C_Uni = selectRegularizationParameter(\"LR\", train_data, 1, 1, 5, False, -3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "< Cross Validation 0.001 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.6846  0.68635 0.68665 0.68405 0.68395]\n",
      "Avg Train Accuracy =  0.68512\n",
      "Logistic Regression Cross Validation Accuracy =  [0.6748 0.678  0.6572 0.667  0.6528]\n",
      "Avg Validation Accuracy =  0.6659599999999999\n",
      "< Cross Validation 0.01 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.71655 0.71825 0.71615 0.7131  0.71305]\n",
      "Avg Train Accuracy =  0.71542\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7028 0.7046 0.6872 0.6936 0.6768]\n",
      "Avg Validation Accuracy =  0.693\n",
      "< Cross Validation 0.1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.81565 0.81835 0.8175  0.816   0.8137 ]\n",
      "Avg Train Accuracy =  0.81624\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7862 0.7826 0.7672 0.7694 0.7676]\n",
      "Avg Validation Accuracy =  0.7746\n",
      "< Cross Validation 1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.9329  0.9363  0.93415 0.9335  0.9337 ]\n",
      "Avg Train Accuracy =  0.9341100000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8456 0.836  0.8378 0.8404 0.8388]\n",
      "Avg Validation Accuracy =  0.83972\n",
      "< Cross Validation 10 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.99915 0.99885 0.99885 0.9989  0.99905]\n",
      "Avg Train Accuracy =  0.99896\n",
      "Logistic Regression Cross Validation Accuracy =  [0.868  0.848  0.8552 0.8642 0.8636]\n",
      "Avg Validation Accuracy =  0.8598000000000001\n",
      "< Cross Validation 100 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8702 0.8502 0.8568 0.8664 0.8636]\n",
      "Avg Validation Accuracy =  0.86144\n",
      "< Cross Validation 1000 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8688 0.85   0.8578 0.8662 0.8648]\n",
      "Avg Validation Accuracy =  0.86152\n"
     ]
    }
   ],
   "source": [
    "LR_C_Bi = selectRegularizationParameter(\"LR\", train_data, 2, 2, 5, False, -3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "< Cross Validation 0.001 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.8591  0.8583  0.85695 0.8574  0.85065]\n",
      "Avg Train Accuracy =  0.85648\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7268 0.733  0.7176 0.72   0.7014]\n",
      "Avg Validation Accuracy =  0.7197600000000001\n",
      "< Cross Validation 0.01 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.86535 0.86375 0.8619  0.8614  0.85775]\n",
      "Avg Train Accuracy =  0.8620300000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.729  0.739  0.7166 0.722  0.7024]\n",
      "Avg Validation Accuracy =  0.7218\n",
      "< Cross Validation 0.1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.9001  0.8997  0.89715 0.8955  0.8924 ]\n",
      "Avg Train Accuracy =  0.8969700000000002\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7454 0.7552 0.7412 0.739  0.7252]\n",
      "Avg Validation Accuracy =  0.7412\n",
      "< Cross Validation 1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.98035 0.98145 0.9811  0.9785  0.97925]\n",
      "Avg Train Accuracy =  0.9801300000000002\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7958 0.7972 0.7884 0.7874 0.7808]\n",
      "Avg Validation Accuracy =  0.7899200000000001\n",
      "< Cross Validation 10 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.818  0.8134 0.8162 0.8124 0.8166]\n",
      "Avg Validation Accuracy =  0.81532\n",
      "< Cross Validation 100 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8206 0.8154 0.8224 0.8202 0.8242]\n",
      "Avg Validation Accuracy =  0.8205600000000001\n",
      "< Cross Validation 1000 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8218 0.8162 0.8264 0.8224 0.826 ]\n",
      "Avg Validation Accuracy =  0.82256\n"
     ]
    }
   ],
   "source": [
    "LR_C_Tri = selectRegularizationParameter(\"LR\", train_data, 3, 3, 5, False, -3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "< Cross Validation 0.001 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.67985 0.68155 0.67915 0.67695 0.67765]\n",
      "Avg Train Accuracy =  0.67903\n",
      "Logistic Regression Cross Validation Accuracy =  [0.6872 0.672  0.6662 0.6642 0.6752]\n",
      "Avg Validation Accuracy =  0.67296\n",
      "< Cross Validation 0.01 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.7103  0.7122  0.7108  0.709   0.71045]\n",
      "Avg Train Accuracy =  0.7105500000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7164 0.7014 0.7016 0.6946 0.7028]\n",
      "Avg Validation Accuracy =  0.70336\n",
      "< Cross Validation 0.1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.8045  0.8028  0.80375 0.80375 0.80185]\n",
      "Avg Train Accuracy =  0.8033300000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7936 0.7892 0.783  0.7814 0.788 ]\n",
      "Avg Validation Accuracy =  0.78704\n",
      "< Cross Validation 1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.89885 0.89945 0.89895 0.89735 0.89665]\n",
      "Avg Train Accuracy =  0.89825\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8584 0.8506 0.85   0.8506 0.859 ]\n",
      "Avg Validation Accuracy =  0.85372\n",
      "< Cross Validation 10 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.98095 0.9829  0.98255 0.98225 0.98175]\n",
      "Avg Train Accuracy =  0.9820800000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8794 0.8688 0.8704 0.8746 0.882 ]\n",
      "Avg Validation Accuracy =  0.8750399999999999\n",
      "< Cross Validation 100 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8822 0.8648 0.8726 0.8782 0.8822]\n",
      "Avg Validation Accuracy =  0.876\n",
      "< Cross Validation 1000 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.881  0.8614 0.8702 0.8764 0.8812]\n",
      "Avg Validation Accuracy =  0.8740399999999999\n"
     ]
    }
   ],
   "source": [
    "LR_C_Uni_Bi = selectRegularizationParameter(\"LR\", train_data, 1, 2, 5, False, -3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "< Cross Validation 0.001 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.67925 0.68015 0.6781  0.67565 0.6751 ]\n",
      "Avg Train Accuracy =  0.6776500000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.685  0.666  0.6662 0.6592 0.6744]\n",
      "Avg Validation Accuracy =  0.67016\n",
      "< Cross Validation 0.01 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.70735 0.70875 0.70835 0.7053  0.7075 ]\n",
      "Avg Train Accuracy =  0.7074500000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7132 0.6964 0.6956 0.6888 0.6962]\n",
      "Avg Validation Accuracy =  0.69804\n",
      "< Cross Validation 0.1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.80175 0.79905 0.80125 0.7998  0.79805]\n",
      "Avg Train Accuracy =  0.7999799999999999\n",
      "Logistic Regression Cross Validation Accuracy =  [0.7856 0.7834 0.7722 0.776  0.7802]\n",
      "Avg Validation Accuracy =  0.7794799999999998\n",
      "< Cross Validation 1 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.9085  0.9082  0.9079  0.9072  0.90545]\n",
      "Avg Train Accuracy =  0.9074500000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8556 0.8474 0.8456 0.8456 0.854 ]\n",
      "Avg Validation Accuracy =  0.84964\n",
      "< Cross Validation 10 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.9921  0.99325 0.9931  0.99235 0.9925 ]\n",
      "Avg Train Accuracy =  0.9926600000000001\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8774 0.8686 0.8702 0.8708 0.8826]\n",
      "Avg Validation Accuracy =  0.87392\n",
      "< Cross Validation 100 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8816 0.8724 0.8724 0.8768 0.884 ]\n",
      "Avg Validation Accuracy =  0.87744\n",
      "< Cross Validation 1000 >\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [1. 1. 1. 1. 1.]\n",
      "Avg Train Accuracy =  1.0\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8836 0.873  0.8744 0.8768 0.8832]\n",
      "Avg Validation Accuracy =  0.8782\n"
     ]
    }
   ],
   "source": [
    "LR_C_Uni_Bi = selectRegularizationParameter(\"LR\", train_data, 1, 3, 5, False, -3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6712\n"
     ]
    }
   ],
   "source": [
    "cParameter = [0.001,0.01,0.1,1,10,100,1000]\n",
    "nGram = [\"1 only\", \"2 only\", \"3 only\"]\n",
    "edit_uni = list()\n",
    "edit_bi = list()\n",
    "edit_tri = list()\n",
    "print((\"%.4f\" % LR_C_Uni[0][0]))\n",
    "for i in range(7):\n",
    "    edit_uni.append((\"%.4f\" % LR_C_Uni[i][0]))\n",
    "    edit_bi.append((\"%.4f\" % LR_C_Bi[i][0]))\n",
    "    edit_tri.append((\"%.4f\" % LR_C_Tri[i][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.6712], [0.7019200000000001], [0.7836000000000001], [0.84724], [0.8648399999999998], [0.8505600000000001], [0.8356]]\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Image data cannot be converted to float",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-80-0ad62e10d4cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0max\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubplots\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;31m# We want to show all ticks...\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\matplotlib\\__init__.py\u001b[0m in \u001b[0;36minner\u001b[1;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1808\u001b[0m                         \u001b[1;34m\"the Matplotlib list!)\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlabel_namer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1809\u001b[0m                         RuntimeWarning, stacklevel=2)\n\u001b[1;32m-> 1810\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0max\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1811\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1812\u001b[0m         inner.__doc__ = _add_data_doc(inner.__doc__,\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\matplotlib\\axes\\_axes.py\u001b[0m in \u001b[0;36mimshow\u001b[1;34m(self, X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, **kwargs)\u001b[0m\n\u001b[0;32m   5492\u001b[0m                               resample=resample, **kwargs)\n\u001b[0;32m   5493\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 5494\u001b[1;33m         \u001b[0mim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   5495\u001b[0m         \u001b[0mim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_alpha\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5496\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_clip_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda\\lib\\site-packages\\matplotlib\\image.py\u001b[0m in \u001b[0;36mset_data\u001b[1;34m(self, A)\u001b[0m\n\u001b[0;32m    640\u001b[0m         if (self._A.dtype != np.uint8 and\n\u001b[0;32m    641\u001b[0m                 not np.can_cast(self._A.dtype, float, \"same_kind\")):\n\u001b[1;32m--> 642\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Image data cannot be converted to float\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    643\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    644\u001b[0m         if not (self._A.ndim == 2\n",
      "\u001b[1;31mTypeError\u001b[0m: Image data cannot be converted to float"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAD8CAYAAACVSwr3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADJlJREFUeJzt22GI5Hd9x/H3x1xTaRq1mBXk7jSRXqrXUIhd0hShRkzLJYW7JyJ3EFpL8NAa+0AppFhSiY8aaQXhWnu0EhU0nj6oi5wEtBGLeJoN0ehduLI9bbNEmlPTPBGNod8+mNFO5rt7+7/L7Mwtfb9gYf7/+c3sd4e59/7nv/9LVSFJk1606AEkXX4Mg6TGMEhqDIOkxjBIagyDpGbLMCT5aJKnknxnk/uT5MNJ1pI8luT1sx9T0jwNOWK4HzhwgftvA/aNv44Cf//Cx5K0SFuGoaq+AvzoAksOAR+vkVPAy5K8clYDSpq/XTN4jt3AExPb6+N9359emOQoo6MKrrrqqt9+7WtfO4NvL2kzjzzyyA+qauliHzeLMGSDfRteZ11Vx4HjAMvLy7W6ujqDby9pM0n+41IeN4u/SqwDeye29wBPzuB5JS3ILMKwAvzR+K8TNwPPVFX7GCFp59jyo0SSTwG3ANckWQf+CvglgKr6CHASuB1YA34M/Ml2DStpPrYMQ1Ud2eL+At41s4kkLZxXPkpqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoGhSHJgSRnk6wluXuD+1+V5KEkjyZ5LMntsx9V0rxsGYYkVwDHgNuA/cCRJPunlv0lcKKqbgQOA38360Elzc+QI4abgLWqOldVzwIPAIem1hTwkvHtlwJPzm5ESfM2JAy7gScmttfH+ya9H7gjyTpwEnj3Rk+U5GiS1SSr58+fv4RxJc3DkDBkg301tX0EuL+q9gC3A59I0p67qo5X1XJVLS8tLV38tJLmYkgY1oG9E9t76B8V7gROAFTV14AXA9fMYkBJ8zckDA8D+5Jcl+RKRicXV6bW/CfwZoAkr2MUBj8rSDvUlmGoqueAu4AHgccZ/fXhdJJ7kxwcL3sv8PYk3wI+BbytqqY/bkjaIXYNWVRVJxmdVJzcd8/E7TPAG2Y7mqRF8cpHSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUDApDkgNJziZZS3L3JmvemuRMktNJPjnbMSXN066tFiS5AjgG/D6wDjycZKWqzkys2Qf8BfCGqno6ySu2a2BJ22/IEcNNwFpVnauqZ4EHgENTa94OHKuqpwGq6qnZjilpnoaEYTfwxMT2+njfpOuB65N8NcmpJAc2eqIkR5OsJlk9f/78pU0sadsNCUM22FdT27uAfcAtwBHgH5O8rD2o6nhVLVfV8tLS0sXOKmlOhoRhHdg7sb0HeHKDNZ+rqp9V1XeBs4xCIWkHGhKGh4F9Sa5LciVwGFiZWvPPwJsAklzD6KPFuVkOKml+tgxDVT0H3AU8CDwOnKiq00nuTXJwvOxB4IdJzgAPAX9eVT/crqElba9UTZ8umI/l5eVaXV1dyPeW/r9I8khVLV/s47zyUVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUjMoDEkOJDmbZC3J3RdY95YklWR5diNKmrctw5DkCuAYcBuwHziSZP8G664G/gz4+qyHlDRfQ44YbgLWqupcVT0LPAAc2mDdB4D7gJ/McD5JCzAkDLuBJya218f7fiHJjcDeqvr8hZ4oydEkq0lWz58/f9HDSpqPIWHIBvvqF3cmLwI+BLx3qyeqquNVtVxVy0tLS8OnlDRXQ8KwDuyd2N4DPDmxfTVwA/DlJN8DbgZWPAEp7VxDwvAwsC/JdUmuBA4DKz+/s6qeqaprquraqroWOAUcrKrVbZlY0rbbMgxV9RxwF/Ag8DhwoqpOJ7k3ycHtHlDS/O0asqiqTgInp/bds8naW174WJIWySsfJTWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSMygMSQ4kOZtkLcndG9z/niRnkjyW5EtJXj37USXNy5ZhSHIFcAy4DdgPHEmyf2rZo8ByVf0W8FngvlkPKml+hhwx3ASsVdW5qnoWeAA4NLmgqh6qqh+PN08Be2Y7pqR5GhKG3cATE9vr432buRP4wkZ3JDmaZDXJ6vnz54dPKWmuhoQhG+yrDRcmdwDLwAc3ur+qjlfVclUtLy0tDZ9S0lztGrBmHdg7sb0HeHJ6UZJbgfcBb6yqn85mPEmLMOSI4WFgX5LrklwJHAZWJhckuRH4B+BgVT01+zElzdOWYaiq54C7gAeBx4ETVXU6yb1JDo6XfRD4VeAzSb6ZZGWTp5O0Awz5KEFVnQROTu27Z+L2rTOeS9ICeeWjpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkZlAYkhxIcjbJWpK7N7j/l5N8enz/15NcO+tBJc3PlmFIcgVwDLgN2A8cSbJ/atmdwNNV9evAh4C/nvWgkuZnyBHDTcBaVZ2rqmeBB4BDU2sOAR8b3/4s8OYkmd2YkuZp14A1u4EnJrbXgd/ZbE1VPZfkGeDlwA8mFyU5Chwdb/40yXcuZegFuYapn+cytpNmhZ01706aFeA3LuVBQ8Kw0W/+uoQ1VNVx4DhAktWqWh7w/S8LO2nenTQr7Kx5d9KsMJr3Uh435KPEOrB3YnsP8ORma5LsAl4K/OhSBpK0eEPC8DCwL8l1Sa4EDgMrU2tWgD8e334L8C9V1Y4YJO0MW36UGJ8zuAt4ELgC+GhVnU5yL7BaVSvAPwGfSLLG6Ejh8IDvffwFzL0IO2nenTQr7Kx5d9KscInzxl/skqZ55aOkxjBIarY9DDvpcuoBs74nyZkkjyX5UpJXL2LOiXkuOO/EurckqSQL+zPbkFmTvHX8+p5O8sl5zzg1y1bvhVcleSjJo+P3w+2LmHM8y0eTPLXZdUEZ+fD4Z3ksyeu3fNKq2rYvRicr/x14DXAl8C1g/9SaPwU+Mr59GPj0ds70Amd9E/Ar49vvXNSsQ+cdr7sa+ApwCli+XGcF9gGPAr823n7F5fzaMjqp987x7f3A9xY47+8Brwe+s8n9twNfYHS90c3A17d6zu0+YthJl1NvOWtVPVRVPx5vnmJ0TceiDHltAT4A3Af8ZJ7DTRky69uBY1X1NEBVPTXnGScNmbeAl4xvv5R+bc/cVNVXuPB1Q4eAj9fIKeBlSV55oefc7jBsdDn17s3WVNVzwM8vp563IbNOupNRhRdly3mT3AjsrarPz3OwDQx5ba8Hrk/y1SSnkhyY23TdkHnfD9yRZB04Cbx7PqNdkot9bw+6JPqFmNnl1HMweI4kdwDLwBu3daILu+C8SV7E6H+6vm1eA13AkNd2F6OPE7cwOhL71yQ3VNV/b/NsGxky7xHg/qr6myS/y+g6nhuq6n+2f7yLdtH/xrb7iGEnXU49ZFaS3Aq8DzhYVT+d02wb2Wreq4EbgC8n+R6jz5YrCzoBOfR98Lmq+llVfRc4yygUizBk3juBEwBV9TXgxYz+g9XlaNB7+3m2+aTILuAccB3/dxLnN6fWvIvnn3w8saATOENmvZHRSal9i5jxYuedWv9lFnfycchrewD42Pj2NYwOfV9+Gc/7BeBt49uvG/9DywLfD9ey+cnHP+T5Jx+/seXzzWHg24F/G/+Det94372MfuPCqLSfAdaAbwCvWeCLu9WsXwT+C/jm+GtlUbMOmXdq7cLCMPC1DfC3wBng28Dhy/m1ZfSXiK+Oo/FN4A8WOOungO8DP2N0dHAn8A7gHROv7bHxz/LtIe8DL4mW1Hjlo6TGMEhqDIOkxjBIagyDpMYwSGoMg6TmfwEval/UlBeDXwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(LR_C_Uni)\n",
    "\n",
    "result = numpy.array([edit_uni, edit_bi, edit_tri])\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "im = ax.imshow(result)\n",
    "\n",
    "# We want to show all ticks...\n",
    "ax.set_xticks(numpy.arange(len(cParameter)))\n",
    "ax.set_yticks(numpy.arange(len(nGram)))\n",
    "# ... and label them with the respective list entries\n",
    "ax.set_xticklabels(cParameter)\n",
    "ax.set_yticklabels(nGram)\n",
    "\n",
    "# Rotate the tick labels and set their alignment.\n",
    "plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "         rotation_mode=\"anchor\")\n",
    "\n",
    "# Loop over data dimensions and create text annotations.\n",
    "for i in range(len(nGram)):\n",
    "    for j in range(len(cParameter)):\n",
    "        text = ax.text(j, i, result[i, j],\n",
    "                       ha=\"center\", va=\"center\", color=\"w\")\n",
    "\n",
    "ax.set_title(\"Harvest of local farmers (in tons/year)\")\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. No data processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in train_data]\n",
    "train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data processing Test, Unigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Train Accuracy =  [0.87515 0.87805 0.87735 0.87565 0.87645]\n",
      "Avg Train Accuracy =  0.87653\n",
      "Logistic Regression Cross Validation Accuracy =  [0.8506 0.8454 0.8444 0.844  0.8518]\n",
      "Avg Validation Accuracy =  0.84724\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([0.99115705, 0.99589705, 1.1870811 , 0.99713731, 0.96800184]),\n",
       " 'score_time': array([0.00500417, 0.00402164, 0.00398922, 0.00398922, 0.00399065]),\n",
       " 'test_score': array([0.8506, 0.8454, 0.8444, 0.844 , 0.8518]),\n",
       " 'train_score': array([0.87515, 0.87805, 0.87735, 0.87565, 0.87645])}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"No data processing Test, Unigram\")\n",
    "crossValidation(\"LR\", X,train_y, 5, 1.0, True)\n",
    "#LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crossValidation(\"LR\", X,train_y, 5, 1.0, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in train_data]\n",
    "train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data processing Test, Bigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9329  0.9363  0.93415 0.9335  0.9337 ]\n",
      "Avg Train Accuracy =  0.9341100000000001\n",
      "LR Cross Validation Accuracy =  [0.8456 0.836  0.8378 0.8404 0.8388]\n",
      "Avg Validation Accuracy =  0.83972\n"
     ]
    }
   ],
   "source": [
    "print(\"No data processing Test, Bigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in train_data]\n",
    "train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(3, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data processing Test, Trigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.98035 0.98145 0.9811  0.9785  0.97925]\n",
      "Avg Train Accuracy =  0.9801300000000002\n",
      "LR Cross Validation Accuracy =  [0.7958 0.7972 0.7884 0.7874 0.7808]\n",
      "Avg Validation Accuracy =  0.7899200000000001\n"
     ]
    }
   ],
   "source": [
    "print(\"No data processing Test, Trigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Allgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in train_data]\n",
    "train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No data processing Test, Allgram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9085  0.9082  0.9079  0.9072  0.90545]\n",
      "Avg Train Accuracy =  0.9074500000000001\n",
      "LR Cross Validation Accuracy =  [0.8556 0.8474 0.8456 0.8456 0.854 ]\n",
      "Avg Validation Accuracy =  0.84964\n"
     ]
    }
   ],
   "source": [
    "print(\"No data processing Test, Allgram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. TF * IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in train_data]\n",
    "train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=True)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF * IDF, Unigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.98965 0.9915  0.99145 0.99045 0.9902 ]\n",
      "Avg Train Accuracy =  0.9906499999999999\n",
      "LR Cross Validation Accuracy =  [0.8648 0.8318 0.8442 0.8542 0.8648]\n",
      "Avg Validation Accuracy =  0.85196\n"
     ]
    }
   ],
   "source": [
    "print(\"TF * IDF, Unigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Extending Abbr. and All Lower Cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start extending sentences\n",
      "[=================== ] 99%\n",
      "Complete extending sentences\n"
     ]
    }
   ],
   "source": [
    "extended = extendSentences(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extended]\n",
    "train_y=[comment[1] for comment in extended]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extending Abbr. and Unigram Test\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.87455 0.87815 0.8771  0.87505 0.87445]\n",
      "Avg Train Accuracy =  0.87586\n",
      "LR Cross Validation Accuracy =  [0.85   0.844  0.8422 0.845  0.8522]\n",
      "Avg Validation Accuracy =  0.8466799999999999\n"
     ]
    }
   ],
   "source": [
    "print(\"Extending Abbr. and Unigram Test\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extended]\n",
    "train_y=[comment[1] for comment in extended]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extending Abbr. and Bigram Test\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9297  0.93135 0.9309  0.92865 0.9294 ]\n",
      "Avg Train Accuracy =  0.93\n",
      "LR Cross Validation Accuracy =  [0.8438 0.8364 0.8382 0.8398 0.836 ]\n",
      "Avg Validation Accuracy =  0.83884\n"
     ]
    }
   ],
   "source": [
    "print(\"Extending Abbr. and Bigram Test\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extended]\n",
    "train_y=[comment[1] for comment in extended]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(3, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extending Abbr. and Trigram Test\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9775  0.97865 0.9776  0.9758  0.97615]\n",
      "Avg Train Accuracy =  0.97714\n",
      "LR Cross Validation Accuracy =  [0.804  0.7966 0.7928 0.7926 0.7892]\n",
      "Avg Validation Accuracy =  0.79504\n"
     ]
    }
   ],
   "source": [
    "print(\"Extending Abbr. and Trigram Test\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4 All-gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extended]\n",
    "train_y=[comment[1] for comment in extended]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extending Abbr. and Allgram Test\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.906   0.90595 0.9065  0.906   0.9046 ]\n",
      "Avg Train Accuracy =  0.90581\n",
      "LR Cross Validation Accuracy =  [0.853  0.8476 0.847  0.8456 0.8534]\n",
      "Avg Validation Accuracy =  0.84932\n"
     ]
    }
   ],
   "source": [
    "print(\"Extending Abbr. and Allgram Test\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 Extended Abbr. Without Stopwords and Punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start removing punctuation and stopwords\n",
      "[=================== ] 99%\n",
      "Complete removing punctuation and stopwords\n"
     ]
    }
   ],
   "source": [
    "extend_withoutStopwordsAndPunctions = removePunctuationAndStopwords(extended, additionalStopwords=['br'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extend_withoutStopwordsAndPunctions]\n",
    "train_y=[comment[1] for comment in extend_withoutStopwordsAndPunctions]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations, unigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9114  0.91475 0.91325 0.91255 0.9107 ]\n",
      "Avg Train Accuracy =  0.91253\n",
      "LR Cross Validation Accuracy =  [0.8634 0.8574 0.8526 0.8584 0.8676]\n",
      "Avg Validation Accuracy =  0.8598800000000001\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations, unigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extend_withoutStopwordsAndPunctions]\n",
    "train_y=[comment[1] for comment in extend_withoutStopwordsAndPunctions]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations, Bigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9824  0.9836  0.9823  0.9816  0.98215]\n",
      "Avg Train Accuracy =  0.98241\n",
      "LR Cross Validation Accuracy =  [0.7954 0.7806 0.7828 0.797  0.7932]\n",
      "Avg Validation Accuracy =  0.7898000000000001\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations, Bigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extend_withoutStopwordsAndPunctions]\n",
    "train_y=[comment[1] for comment in extend_withoutStopwordsAndPunctions]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(3, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations, Trigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9996  0.99965 0.9995  0.9996  0.9995 ]\n",
      "Avg Train Accuracy =  0.9995700000000001\n",
      "LR Cross Validation Accuracy =  [0.683  0.6686 0.6474 0.6776 0.6642]\n",
      "Avg Validation Accuracy =  0.66816\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations, Trigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.4 Allgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in extend_withoutStopwordsAndPunctions]\n",
    "train_y=[comment[1] for comment in extend_withoutStopwordsAndPunctions]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations, Allgram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.93305 0.935   0.93425 0.9334  0.9319 ]\n",
      "Avg Train Accuracy =  0.93352\n",
      "LR Cross Validation Accuracy =  [0.8582 0.8536 0.8448 0.8524 0.861 ]\n",
      "Avg Validation Accuracy =  0.8539999999999999\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations, Allgram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 Extended Abbr. Without Stopwords and Punctuations and Lemmatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start lemmatizing\n",
      "[=================== ] 99%\n",
      "Complete lemmatizing\n"
     ]
    }
   ],
   "source": [
    "lemmatized = lemmatize(extend_withoutStopwordsAndPunctions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in lemmatized]\n",
    "train_y=[comment[1] for comment in lemmatized]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Unigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.90855 0.91175 0.91045 0.90895 0.9082 ]\n",
      "Avg Train Accuracy =  0.90958\n",
      "LR Cross Validation Accuracy =  [0.8626 0.8558 0.85   0.856  0.8606]\n",
      "Avg Validation Accuracy =  0.857\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Unigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2 Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in lemmatized]\n",
    "train_y=[comment[1] for comment in lemmatized]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Bigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.97755 0.9785  0.977   0.97685 0.97875]\n",
      "Avg Train Accuracy =  0.97773\n",
      "LR Cross Validation Accuracy =  [0.7924 0.77   0.7716 0.7922 0.7932]\n",
      "Avg Validation Accuracy =  0.78388\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Bigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3 Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in lemmatized]\n",
    "train_y=[comment[1] for comment in lemmatized]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(3, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Trigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.999   0.99885 0.99895 0.99915 0.9988 ]\n",
      "Avg Train Accuracy =  0.9989500000000001\n",
      "LR Cross Validation Accuracy =  [0.663  0.6524 0.64   0.666  0.6526]\n",
      "Avg Validation Accuracy =  0.6548\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations and Lemmatized, Trigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.4 Allgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in lemmatized]\n",
    "train_y=[comment[1] for comment in lemmatized]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords and Punctuations and Lemmatized, allgram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.93105 0.9331  0.93175 0.9306  0.9293 ]\n",
      "Avg Train Accuracy =  0.93116\n",
      "LR Cross Validation Accuracy =  [0.855  0.85   0.8448 0.8508 0.8586]\n",
      "Avg Validation Accuracy =  0.8518399999999999\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords and Punctuations and Lemmatized, allgram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6 Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start removing duplicates\n",
      "[=================== ] 99%\n",
      "Complete removing duplicates\n"
     ]
    }
   ],
   "source": [
    "noDuplicates = removeDuplicates(lemmatized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.1 Unigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in noDuplicates]\n",
    "train_y=[comment[1] for comment in noDuplicates]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, unigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9131  0.91605 0.9146  0.9125  0.91075]\n",
      "Avg Train Accuracy =  0.9134\n",
      "LR Cross Validation Accuracy =  [0.8662 0.8652 0.8654 0.8668 0.8724]\n",
      "Avg Validation Accuracy =  0.8672000000000001\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, unigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.2 Bigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in noDuplicates]\n",
    "train_y=[comment[1] for comment in noDuplicates]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, bigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9882  0.9881  0.98885 0.9876  0.9883 ]\n",
      "Avg Train Accuracy =  0.9882099999999999\n",
      "LR Cross Validation Accuracy =  [0.7654 0.7532 0.7556 0.761  0.7682]\n",
      "Avg Validation Accuracy =  0.76068\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, bigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.3 Trigram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in noDuplicates]\n",
    "train_y=[comment[1] for comment in noDuplicates]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(3, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, trigram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.99935 0.9992  0.99925 0.99955 0.9994 ]\n",
      "Avg Train Accuracy =  0.99935\n",
      "LR Cross Validation Accuracy =  [0.637  0.6176 0.6068 0.628  0.6316]\n",
      "Avg Validation Accuracy =  0.6242\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, trigram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.4 Allgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=[comment[0] for comment in noDuplicates]\n",
    "train_y=[comment[1] for comment in noDuplicates]\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n",
    "X = tfidf_transformer.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, alligram\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Train Accuracy =  [0.9381  0.9396  0.93935 0.9377  0.93645]\n",
      "Avg Train Accuracy =  0.9382400000000001\n",
      "LR Cross Validation Accuracy =  [0.859  0.8566 0.8578 0.8598 0.8624]\n",
      "Avg Validation Accuracy =  0.8591200000000001\n"
     ]
    }
   ],
   "source": [
    "print(\"Extended Abbr. Without Stopwords, Punctuations and duplicates, and Lemmatized, alligram\")\n",
    "LogReg(X,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVM(X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Determine Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selectLGParameters(dataSet,leastNgram, mostNgram, tfidf):\n",
    "    bestScore = 0\n",
    "    bestC = 0\n",
    "    output = list()\n",
    "    for i in range(100):\n",
    "        \n",
    "        #crossValidation(\"LR\", X,y_train, fold, c, train_score):\n",
    "        \n",
    "        train_set,valid_set=sklearn.model_selection.train_test_split(dataSet,train_size=0.8,test_size=0.2,shuffle=True)\n",
    "        \n",
    "        train_x=[comment[0] for comment in train_set]\n",
    "        train_y=[comment[1] for comment in train_set]\n",
    "        valid_x=[comment[0] for comment in valid_set]\n",
    "        valid_y=[comment[1] for comment in valid_set]\n",
    "\n",
    "        vectorizer = CountVectorizer(ngram_range=(leastNgram, mostNgram))\n",
    "        X = vectorizer.fit_transform(train_x)\n",
    "\n",
    "        tfidf_transformer = TfidfTransformer(use_idf=tfidf)\n",
    "        X = tfidf_transformer.fit_transform(X)\n",
    "    \n",
    "        lg=LogisticRegression(C=1)\n",
    "        lg.fit(X,train_y)\n",
    "        X_valid=vectorizer.transform(valid_x)\n",
    "        valid_pred=lg.predict(X_valid)\n",
    "        score = sklearn.metrics.accuracy_score(valid_y, valid_pred)\n",
    "        if score > bestScore:\n",
    "            bestC = i\n",
    "        print(\"C = \", 1+i, \", Accuracy = \", score)\n",
    "        output.append([bestC, bestScore])\n",
    "    print(\"Best LR C = \", bestC, \"\\nAccuracy = \", bestScore)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start extending sentences\n",
      "[=================== ] 99%\n",
      "Complete extending sentences\n",
      "Start removing punctuation and stopwords\n",
      "[=================== ] 99%\n",
      "Complete removing punctuation and stopwords\n",
      "Start lemmatizing\n",
      "[=================== ] 99%\n",
      "Complete lemmatizing\n",
      "Start removing duplicates\n",
      "[=================== ] 99%\n",
      "Complete removing duplicates\n"
     ]
    }
   ],
   "source": [
    "extended = extendSentences(train_data)\n",
    "extend_withoutStopwordsAndPunctions = removePunctuationAndStopwords(extended, additionalStopwords=['br'])\n",
    "lemmatized = lemmatize(extend_withoutStopwordsAndPunctions)\n",
    "noDuplicates = removeDuplicates(lemmatized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C =  1 , Accuracy =  0.846\n",
      "C =  2 , Accuracy =  0.8706\n",
      "C =  3 , Accuracy =  0.8696\n",
      "C =  4 , Accuracy =  0.8794\n",
      "C =  5 , Accuracy =  0.8846\n",
      "C =  6 , Accuracy =  0.877\n",
      "C =  7 , Accuracy =  0.8858\n",
      "C =  8 , Accuracy =  0.8834\n",
      "C =  9 , Accuracy =  0.8828\n",
      "C =  10 , Accuracy =  0.8872\n",
      "C =  11 , Accuracy =  0.8798\n",
      "C =  12 , Accuracy =  0.8888\n",
      "C =  13 , Accuracy =  0.8858\n",
      "C =  14 , Accuracy =  0.8822\n",
      "C =  15 , Accuracy =  0.887\n",
      "C =  16 , Accuracy =  0.8838\n",
      "C =  17 , Accuracy =  0.8924\n",
      "C =  18 , Accuracy =  0.8844\n",
      "C =  19 , Accuracy =  0.8774\n",
      "C =  20 , Accuracy =  0.8896\n",
      "C =  21 , Accuracy =  0.878\n",
      "C =  22 , Accuracy =  0.8882\n",
      "C =  23 , Accuracy =  0.889\n",
      "C =  24 , Accuracy =  0.8838\n",
      "C =  25 , Accuracy =  0.8968\n",
      "C =  26 , Accuracy =  0.8842\n",
      "C =  27 , Accuracy =  0.8864\n",
      "C =  28 , Accuracy =  0.8848\n",
      "C =  29 , Accuracy =  0.882\n",
      "C =  30 , Accuracy =  0.8802\n",
      "C =  31 , Accuracy =  0.884\n",
      "C =  32 , Accuracy =  0.8944\n",
      "C =  33 , Accuracy =  0.8908\n",
      "C =  34 , Accuracy =  0.8846\n",
      "C =  35 , Accuracy =  0.8846\n",
      "C =  36 , Accuracy =  0.8904\n",
      "C =  37 , Accuracy =  0.8836\n",
      "C =  38 , Accuracy =  0.8846\n",
      "C =  39 , Accuracy =  0.8918\n",
      "C =  40 , Accuracy =  0.8894\n",
      "C =  41 , Accuracy =  0.8892\n",
      "C =  42 , Accuracy =  0.8842\n",
      "C =  43 , Accuracy =  0.8948\n",
      "C =  44 , Accuracy =  0.8928\n",
      "C =  45 , Accuracy =  0.8898\n",
      "C =  46 , Accuracy =  0.8866\n",
      "C =  47 , Accuracy =  0.8928\n",
      "C =  48 , Accuracy =  0.8874\n",
      "C =  49 , Accuracy =  0.887\n",
      "C =  50 , Accuracy =  0.8912\n",
      "C =  51 , Accuracy =  0.8962\n",
      "C =  52 , Accuracy =  0.8958\n",
      "C =  53 , Accuracy =  0.8876\n",
      "C =  54 , Accuracy =  0.8932\n",
      "C =  55 , Accuracy =  0.8898\n",
      "C =  56 , Accuracy =  0.8846\n",
      "C =  57 , Accuracy =  0.89\n",
      "C =  58 , Accuracy =  0.8864\n",
      "C =  59 , Accuracy =  0.8898\n",
      "C =  60 , Accuracy =  0.8906\n",
      "C =  61 , Accuracy =  0.887\n",
      "C =  62 , Accuracy =  0.885\n",
      "C =  63 , Accuracy =  0.896\n",
      "C =  64 , Accuracy =  0.8886\n",
      "C =  65 , Accuracy =  0.8872\n",
      "C =  66 , Accuracy =  0.8928\n",
      "C =  67 , Accuracy =  0.8922\n",
      "C =  68 , Accuracy =  0.886\n",
      "C =  69 , Accuracy =  0.893\n",
      "C =  70 , Accuracy =  0.8896\n",
      "C =  71 , Accuracy =  0.8904\n",
      "C =  72 , Accuracy =  0.884\n",
      "C =  73 , Accuracy =  0.8914\n",
      "C =  74 , Accuracy =  0.8934\n",
      "C =  75 , Accuracy =  0.8806\n",
      "C =  76 , Accuracy =  0.881\n",
      "C =  77 , Accuracy =  0.8932\n",
      "C =  78 , Accuracy =  0.8846\n",
      "C =  79 , Accuracy =  0.889\n",
      "C =  80 , Accuracy =  0.8896\n",
      "C =  81 , Accuracy =  0.8842\n",
      "C =  82 , Accuracy =  0.8858\n",
      "C =  83 , Accuracy =  0.8874\n",
      "C =  84 , Accuracy =  0.8912\n",
      "C =  85 , Accuracy =  0.8904\n",
      "C =  86 , Accuracy =  0.8954\n",
      "C =  87 , Accuracy =  0.8852\n",
      "C =  88 , Accuracy =  0.8884\n",
      "C =  89 , Accuracy =  0.8866\n",
      "C =  90 , Accuracy =  0.887\n",
      "C =  91 , Accuracy =  0.8948\n",
      "C =  92 , Accuracy =  0.8822\n",
      "C =  93 , Accuracy =  0.8876\n",
      "C =  94 , Accuracy =  0.8846\n",
      "C =  95 , Accuracy =  0.8844\n",
      "C =  96 , Accuracy =  0.888\n",
      "C =  97 , Accuracy =  0.8896\n",
      "C =  98 , Accuracy =  0.8788\n",
      "C =  99 , Accuracy =  0.8824\n",
      "C =  100 , Accuracy =  0.8872\n",
      "Best LR C =  99 \n",
      "Accuracy =  0\n"
     ]
    }
   ],
   "source": [
    "result = selectLGParameters(train_data,1, 1, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Competition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LR(X,y_train,x_valid,y_valid, c):\n",
    "    lg=LogisticRegression(C=c)\n",
    "    lg.fit(X,y_train)\n",
    "    X_valid=vectorizer.transform(x_valid)\n",
    "    valid_pred=lg.predict(X_valid)\n",
    "    print (\"LR Accuracy: \", sklearn.metrics.accuracy_score(y_valid, valid_pred))\n",
    "    print (sklearn.metrics.confusion_matrix(y_valid, valid_pred, labels=[1, 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_data=load_test()\n",
    "#extended = extendSentences(test_data)\n",
    "#extend_withoutStopwordsAndPunctions = removePunctuationAndStopwords(extended, additionalStopwords=['br'])\n",
    "#lemmatized = lemmatize(extend_withoutStopwordsAndPunctions)\n",
    "#noDuplicates = removeDuplicates(lemmatized)\n",
    "\n",
    "total_train_x=[comment[0] for comment in train_data]\n",
    "total_train_y=[comment[1] for comment in train_data]\n",
    "\n",
    "test_x=[comment[0] for comment in test_data]\n",
    "test_id=[comment[1] for comment in test_data]\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 3))\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X=vectorizer.fit_transform(total_train_x)\n",
    "train_X=tfidf_transformer.fit_transform(train_X)\n",
    "lg=LogisticRegression(C=100)\n",
    "lg.fit(train_X,total_train_y)\n",
    "X_valid=vectorizer.transform(test_x)\n",
    "X_valid=train_X=tfidf_transformer.fit_transform(X_valid)\n",
    "y_pred=lg.predict(X_valid)\n",
    "\n",
    "\n",
    "df=pd.DataFrame(data={'Id':test_id,'Category':y_pred},columns=['Id','Category'])\n",
    "df.to_csv('D:\\\\McGill\\\\19Fall\\\\COMP 551\\\\Projects\\\\Project2\\\\comp-551-imbd-sentiment-classification\\\\LR.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
